<?php 
	/**
	 * 自动注册笑话id
	 */
	 //将某一天的数据导出成sql文件
	function jokeDataDump($ymd="")
	{
		global $conn;
		if (empty($ymd)) {
			$ymd = date('Y-m-d',time()-24*60*60);
			$ymd_begin = strtotime(date("Y-m-d H:00:00", time()-60*60));
			$ymd_end = strtotime(date("Y-m-d H:59:59", time()-60*60));
		}else{
			$ymd_begin = strtotime($ymd." 00:00:00");
			$ymd_end = strtotime($ymd." 23:59:59");
		}
		echo $ymd;
		$datafile = ROOT_PATH."/joke/data/joke_".$ymd.".sql";
		file_put_contents($datafile, ""); //清空
		$tblName = 'jiaxin_weibo_timeline';
		$sql = "select * from $tblName where crawltime>='$ymd_begin' and crawltime<='$ymd_end' ";
		$query = $conn->query($sql);
		while($row = $conn->fetch_assoc()){
			$row['id'] = null;
			$arr = $row;
			$sql_insert = Sql::insert($arr, $tblName);
			//echo $sql_insert;exit;
			file_put_contents($datafile, $sql_insert."; \n", FILE_APPEND);
		}
		return $datafile;
	}
	//将某个文件上传至ftp服务器
	function jokeDataUpload($datafile)
	{
		if(!$handle = fopen($datafile, 'rb')){
			printlog("error opening ".$datafile, "ftp_upload");
		}
		
		$size = filesize($datafile);
		$contents = file_get_contents($datafile);
		$file_base_name = basename($datafile);
		while(true){
			$remote_file = "ftp://".FTP_USERNAME.":".FTP_PASSWORD."@".FTP_SERVER.":".FTP_PORT.FTP_ROMATE_PATH.$file_base_name;
			//echo $remote_file;exit;
			$len = file_put_contents($remote_file, $contents);
			if($len == $size){
				$title = "$datafile 上传到ftp成功， 上传文件大小为 $size ";
				sendMailEx($title, $title);
				break;
			}
			else{
				printlog("upload $datafile error,retry....", "ftp_upload");
				unlink($remote_file);
			}
		}
	}
	//统计某个日期抓取的数据
	function sendJokeInfoMail($ymd="")
	{
		global $conn;
		if (empty($ymd)) {
			//$ymd = date('Y-m-d',time()-60*60);
			$ymd_begin = strtotime(date("Y-m-d H:00:00", time()-60*60));
			$ymd_end = strtotime(date("Y-m-d H:59:59", time()-60*60));
		}else{
			$ymd_begin = strtotime($ymd." 00:00:00");
			$ymd_end = strtotime($ymd." 23:59:59");
		}
		$sql = "select count(*) as total ,site from jiaxin_weibo_timeline where crawltime>=$ymd_begin and crawltime<=$ymd_end  group by site"; 
		$query = $conn->query($sql);
		$body = '';
		$total = 0 ;
		while($row = $conn->fetch_array()){
			$total += $row['total'];
			if($row['site'] == 1){
				$body .= '昨天，网站haha.mx共抓取'.$row['total']."<br>";
			}elseif ($row['site'] == 2) {
				$body .= '昨天，网站360.cn共抓取'.$row['total']."<br>";
			}else{
				$body .= '昨天，网站day.2345.com共抓取'.$row['total']."<br>";
			}
		}
		//发送邮件
		$title = "共抓取 $total 条笑话 ";
		$body = $body.$title ;
		
		sendMailEx($title, $body);
	}
	//判断笑话抓下来的页面是否包含正常的uid
	function checkJokeIdExists($page)
	{
		if(strlen($page)<200)
			return false;
		else
			return true;
	}

	//糗百抓取笑话主程
	function qiubai_crawl()
	{
		$opt_arr = array(CURLOPT_REFERER => "http://search.baidu.com/search",  //referer	
					);
		$i = 1;
		while($i<2000){
			$url = 'http://www.qiushibaike.com/late/page/'.$i;
			$page = makeRequest($url, 
					"GET", 
					"", 
					false, 
					"",
					"",
					""
					);
			if($page == false){ 
				sleep(5);
				continue;
			}
			//printlog("success!!: $page", "qiubai");
			$do = doWithQiuBaiPage($page);
			if($do == 2){
				echo "over";
				exit;
			}
			$i++;
		}
	}
	//处理入库
	function doWithQiuBaiPage($page){
		global $conn;
		preg_match_all('%<div class="content".*? title=".*?>([\s\S]*?)?</div>%i',$page,$matches);
		$have = 0 ;
		$tblName = 'jiaxin_joke';
		foreach($matches[1] as $ke=>$joke){
			if($joke ==''){ break; }
			
			preg_match('%<a.*?article/(\d+)%i',$joke,$m);
			if(isset($m[1])){
				if(haveNoThis($m[1])){
					
					if(preg_match('%<img.*?%i',$joke)){ continue;}
					$content = strip_tags($joke);
					$content = str_replace('糗事百科','我的过去',$content);
					$arr = array(
						"joke_id" => $m[1],
						"content" => $content,
						"site" => 'qiubai',
						"ctime" => time(),
						);
					$sql = Sql::insert($arr, $tblName);
					$conn->query($sql);
				}else{
					$have++;
				}
			}
		}
		if($have >3){
				return 2;
		}
		return 1;
	}
	//产生时间戳
	function microtime_int(){
		list($usec, $sec) = explode(" ", microtime());
		return  round((float)($sec.$usec),2)*100;
	}
	function havepass($content)
	{
		$passarray= array('@','via','by','关注','转发','http','星座','水瓶座','双鱼座','白羊座','金牛座',
			'双子座','巨蟹座','狮子座','处女座','天秤座','天蝎座','射手座','摩羯座','淘宝','链接','兼职',
			'哈友','顶','踩','转发','平衡','文革','大陆','台湾','反共','种子','割','城管','五毛党','蒋介石',
			'二奶','共产党','天朝','haha','HAHA');
		$passflag = 0;
		foreach ($passarray as $key => $value) {
			if (strpos($content, $value) !== false) {
				$passflag = 1;
				break;
			}
		}
		return $passflag;
	}
	//haha.mx抓取笑话主程
	function haha_crawl()
	{
		$opt_arr = array(CURLOPT_REFERER => "http://search.baidu.com/search",  //referer	
					);
		$i = 1;
		while($i<2){
			$url = 'http://www.haha.mx/good/day/'.$i;
			$page = makeRequest($url, 
					"GET", 
					"", 
					false, 
					"",
					"",
					$opt_arr
					);
			//printlog("success!!: $page", "hahamx");

			if($page == false){ 
				sleep(5);
				continue;
			}
			$do = doWithHaHaPage($page);
			if($do == 2){
				echo "over";
				exit;
			}
			$i++;
			sleep(5);
		}
	}
	//处理入库
	function doWithHaHaPage($page){
		global $conn;
		$tblName = 'jiaxin_weibo_timeline';
		$page = str_replace(array(" ","\t","\n","\r","&nbsp;"), "", $page);
		//file_put_contents('aaa.txt', $page);
		//exit;
		$have = 0;
		preg_match_all('%<divclass="blockjoke-item".*?>(.*?)</div></div></div>%i', $page,$match);
		if (isset($match[1]) && is_array($match[1])) {
			foreach ($match[1] as $key => $value) {
				preg_match('%<pclass="textword-wrap">(.*?)</p>%i', $value,$content);
				if (isset($content[1])) {
					$joke_content = $content[1];
					if (havepass($joke_content)) {
						echo  'passed';
						continue;
					}
					$joke_content = str_replace('绿', '顶',$joke_content );
					$joke_content = str_replace('红', '踩',$joke_content );
					$joke_content = str_replace('##', '',$joke_content );
					$joke_content = str_replace(array(" ","\t","\n","\r","&nbsp;"), "", $joke_content);
					$md5 = md5($joke_content);
				}else{
					$joke_content = '';
				}
				if ($joke_content == '') {
					continue;
				}
				
				$arr = array(
							"uid" => 1,
							"type" => 2,
							"site" => 1,
							"content" => $joke_content,
							"md5" => $md5,
							);
				preg_match('%data="\{\'url\':http://www.haha.mx/joke/(\d+)%i', $value,$matchid);
				if (isset($matchid[1])) {
					$joke_id = $matchid[1];
				}else{
					$joke_id = 0;
				}
				$sql = "select * from $tblName where `site`='haha' and weiboid='$joke_id' limit 1";		
				$exists = $conn->has_record($sql);
				if ($exists) {
					$have++;
					continue;
				}
				if ($have >=3) {
					return 2;
				}
				$arr['weiboid'] = $joke_id;
				preg_match('/data=".*?\'pic\':(.*?)\}/i', $value,$pic);
				if (isset($pic[1])) {
					$bmiddle_pic = $pic[1];
					$picurl = str_replace('middle', 'big', $bmiddle_pic);
					$thumbnail_pic = str_replace('middle', 'small', $bmiddle_pic);
				}else{
					$bmiddle_pic = $picurl = $thumbnail_pic = '';
				}
				$arr['bmiddle_pic'] = $bmiddle_pic;
				$arr['picurl'] = $picurl;
				$arr['thumbnail_pic'] = $thumbnail_pic;
				
				preg_match('%<ahref=\'/user/\d+\'>(.*?)</a>.*?joke-time">(.*?)发布%i', $value,$user);
				
				if (isset($user[1])) {
					$username = $user[1];
					$posttime = strtotime($user[2]);
				}else{
					$username = '网友';
					$posttime = time();
				}
				$arr['username'] = $username;
				$arr['posttime'] = $posttime;
				$arr['crawltime'] = time();
				preg_match('%data="g">(\d+)</a>%i', $value,$dinghot);
				if (isset($dinghot[1])) {
					$dinghotnum = $dinghot[1];
				}
				$arr['dinghot'] = $dinghotnum;
				preg_match('%data="b">(\d+)</a>%i', $value,$cai);
				if (isset($cai[1])) {
					$cainum = $cai[1];
				}
				//$arr['cai'] = $cainum;
				$arr['cai'] = 0;
				$arr['flag'] = 0;
				//var_dump($arr);
				$sql = Sql::insert($arr, $tblName);
				$conn->query($sql);
			}
		}
	}
	//获取最大的id
	function getQihuMaxId()
	{
		global $conn;
		$sql = "select max(joke_id) from jiaxin_joke where site='qihu'";
		$maxId = $conn->result($sql);
		return $maxId;
	}
	//hao.360.cn抓取笑话主程
	function qihu_crawl($begin=1)
	{
		$opt_arr = array(CURLOPT_REFERER => "http://xiaohua.360.cn",  //referer	
					);
		$i =$begin;
		$have = 0 ;
		while(true){
			$url = 'http://xiaohua.hao.360.cn/?id='.$i;
			$page = makeRequest($url, 
					"GET", 
					"", 
					false, 
					"",
					"",
					$opt_arr
					);
			//printlog("success!!: $page", "qiubai");
			if($page == false){ 
				sleep(5);
				continue;
			}
			$do = doWithQihuPage($page);
			//$do = doWithQihuPage($i,$page);
			if($do == 2){
				echo 'over';
				exit;
			}
			
			$i++;
			echo $i."\n";
			sleep(5);
		}
	}
	//处理入库
	function doWithQihuPagebak($id,$page){
		global $conn;
		$html = str_replace(array("\n","\r", " ", "\t"), "", $page);
		file_put_contents('qihu.txt', $html);
		exit;
		preg_match('%<ddid="content"class="details">(.*?)</dd>%i', $html,$match);
		var_dump($match);
		$content = trim($match[1],'<p></p>');
		$content = str_replace('</p><p>','\r\n',$content);
		$md5 = md5($content);
		if(hasHash($md5)){
			return -2;
		}
		$arr = array(
				"joke_id" => $id,
				"content" => $content,
				"md5" => $md5,
				"site" => 'qihu',
				"ctime" => time(),
				);
		var_dump($arr);
		$sql = Sql::insert($arr, $tblName);
		$conn->query($sql);
				
		return true;
	}
	//处理入库
	function doWithQihuPage($page){
		global $conn;
		$user = array('新浪微博','腾讯微博');
		//file_put_contents('aaa.txt', $page);
		//exit;
		preg_match('/var jokes = (.*?);/i', $page,$match);
		$data = json_decode($match[1]);		
		$tblName = 'jiaxin_weibo_timeline';
		$have = 0 ;
		$count = count($data);
		if($count > 0){
			foreach($data as $key=>$value){
				$sql = "select * from $tblName where `site`=2 and weiboid='".$value[0]."' limit 1";		
				$exists = $conn->has_record($sql);
				if ($exists) {
					$have++;
					continue;
				}
				$joke_content = $value[1];
				if ($value[2]!= '#') {
					$joke_content .=','.$value[2];
				}
				if (havepass($joke_content)) {
					echo  'passed';
					continue;
				}
				$joke_content = str_replace('##', '',$joke_content );
				$joke_content = str_replace(array(" ","\t","\n","\r","&nbsp;"), "", $joke_content);
				$md5 = md5($joke_content);
				$arr = array(
							"uid" => 1,
							"type" => 2,
							"site" => 2,
							"content" => $joke_content,
							"md5" => $md5,
							"weiboid"=>$value[0],
							"flag"=>0,
							);
				//var_dump($arr);
				if (isset($value[3])) {
					$arr['bmiddle_pic'] = $value[3];
					$arr['picurl'] =$value[3];
					$arr['thumbnail_pic'] = $value[3];
				}
				$uid = array_rand($user,1);
				$arr['username'] = $user[$uid];
				$arr['posttime'] = time();
				$arr['crawltime'] = time();
				$arr['dinghot'] = rand(200,1000);
		
				$sql = Sql::insert($arr, $tblName);
				$conn->query($sql);
			}
		}
		if($have >5){
				return 2;
		}
		return 1;
	}
	//查看是否有笑话
	function hasHash($value='')
	{
		global $conn;
		$tblName = 'jiaxin_weibo_timeline';
		$sql = "select * from $tblName where md5='$value' limit 1";		
		$exists = $conn->has_record($sql);
		if($exists){
			return true;
		}else{
			return false;
		}
	}
	//查看是否抓取过
	function haveNoThis($id,$site=3){
		global $conn;
		$tblName = 'jiaxin_weibo_timeline';
		$sql = "select * from $tblName where `site`='$site' and weiboid='$id' limit 1";
		
		$exists = $conn->has_record($sql);
		if($exists){
			return false;
		}else{
			return true;
		}
	}
	
	//糗百抓取笑话主程
	function yile_crawl()
	{
		$opt_arr = array(CURLOPT_REFERER => "http://search.baidu.com/search",  //referer	
					);
		$i = 1;
		while($i<20){
			$url = 'http://day.2345.com/gaoxiao_'.$i.'/';
			$page = makeRequest($url, 
					"GET", 
					"", 
					false, 
					"",
					"",
					$opt_arr
					);
			if($page == false){ 
				sleep(5);
				continue;
			}
			//printlog("success!!: $page", "yile");
			$do = doWithYiLePage($page);
			if($do == 2){
				echo "over";
				exit;
			}
			$i++;
			sleep(5);
		}
	}
	//处理入库
	function doWithYiLePage($page){
		global $conn;
		$page = str_replace(array(" ","\t","\n","\r","&nbsp;"), "", $page);
		file_put_contents('yile.txt', $page);
		exit;
		$user = array('新浪微博','腾讯微博');
		preg_match_all('%<divclass="mod_bmt10">.*?<divclass="tb_title"><h4>(.*?)</h4><span>(.*?)</span></div>(.*?)</div><divclass="author">%i',$page,$matches);
		preg_match_all('%<ahref="#"class="voteUp">(.*?)</a><ahref="#"class="voteDown">(.*?)</a>%i',$page,$dingcai);
		preg_match_all('%<ahref="/gaoxiao/(.*?)/"%i',$page,$uids);
		$have = 0 ;
		$tblName = 'jiaxin_weibo_timeline';
		$count = count($matches[1]);
		$jokes = $matches[1];
		$uid = $uids[1];
		$i = 0;
		
		while($i<$count){
			if($jokes[$i] ==''){ break; }
			if(isset($uid[$i])){

				if(haveNoThis($uid[$i],3)){
					$arr = array(
							"uid" => 1,
							"type" => 2,
							"site" => 3,
							"weiboid"=>substr($uid[$i], 6),
							"flag"=>0,
							);
					if (preg_match('/<imgclass="lazyLoad".*?data-url="(.*?)"/i', $matches[3][$i],$pic)) {
						$joke_content = iconv('gb2312', 'UTF-8', $jokes[$i]);
						$md5 = md5($joke_content);
						$arr['content'] = $joke_content;
						$arr['md5'] = $md5;
						$picurl = 'http://day.2345.com/'.$pic[1];
						$arr['bmiddle_pic'] = $picurl;
						$arr['picurl'] =$picurl;
						$arr['thumbnail_pic'] = $picurl;
					} else {
						$joke_content= $jokes[$i]. $matches[3][$i];
						$joke_content = iconv('gb2312', 'UTF-8', $joke_content);
						$md5 = md5($joke_content);
						$arr['content'] = $joke_content;
						$arr['md5'] = $md5;
					}
					$uid = array_rand($user,1);
					$arr['username'] = $user[$uid];
					
					$arr['posttime'] = strtotime(chunk_split($matches[2][$i],10,' '));
					$arr['crawltime'] = time();
					$arr['dinghot'] = $dingcai[1][$i];
					//$arr['cai'] = $dingcai[2][$i];
					
					$sql = Sql::insert($arr, $tblName);
					$conn->query($sql);
				}else{
					$have++;
				}
			}
			$i++;
		}
		if($have >5){
				return 2;
		}
		return 1;
	}
?>